import re
import urllib.parse

from bs4 import BeautifulSoup

class HtmlParser(object):
    def parse(self, pageUrl, htmlCont):
        if pageUrl is None or htmlCont is None:
            return
        soup = BeautifulSoup(htmlCont, 'html.parser', from_encoding='utf-8')
        newUrls = self.getNewUrls(pageUrl, soup)
        newData = self.getNewData(pageUrl, soup)
        return newUrls, newData

    def getNewUrls(self, pageUrl, soup):
        newUrls = set()
        try:
            links = soup.find('div', class_='article-list').findAll('a', href=re.compile(r"weixin_41475710/article/details/"))
            for link in links:
                newUrl = link['href']
                newFullUrl = urllib.parse.urljoin(pageUrl, newUrl)
                newUrls.add(newFullUrl)
        except:
            return None
        return newUrls

    def getNewData(self, pageUrl, soup):
        resData = {}
        resData['url'] = pageUrl
        try:
            titleNode = soup.find('h1', class_='title-article')
            resData['title'] = titleNode.getText()
        except:
            resData['title'] = "没有内容"
        try:
            summaryNode = soup.find('div', class_='htmledit_views')
            resData['summary'] = summaryNode.getText()
        except:
            try:
                titleNode = soup.find('div', class_='markdown_views prism-atelier-sulphurpool-light')
                resData['summary'] = titleNode.getText()
            except:
                resData['summary'] = "没有内容"
        return resData